Step 1: Set up D3 area


In [1]:
from IPython.core.display import display, HTML
from string import Template
import pandas as pd
import json, random

In [2]:
%%javascript
require.config({paths: {d3: "https://d3js.org/d3.v4.min"}});
require(['d3'], function(d3) {
    window.d3 = d3;
})



In [3]:
html_template = Template('''
<svg id="graph-div"></div>
<script> $js_text </script>
''')

In [17]:
js_text_template = Template('''
var data = $data;
var itemWidth = 50;
var itemHeight = 50;

var chart = d3.select('#graph-div')
    .attr('width', 800)
    .attr('height', itemHeight*4);

function update() {
    var row = chart.selectAll('.item')
        .data(data);
    
    var row2 = row.enter()
        .append('g')
        .attr("class", "item");
        
    row2.merge(row)
        .attr('transform', function(d, i){
            return 'translate(' + i*(itemWidth+5) + ', ' + itemHeight + ')rotate(' + d + ', ' + itemWidth*0.5 + ', ' + itemHeight*0.5 + ')';
        });

    row2.append('rect')
        .attr("width", itemWidth)
        .attr("height", itemHeight)
        .attr("fill", "#9999FF")
        .attr("stroke", "#5555AA");

    row2.append("text")
        .text("Hello");
}

update();
''')

In [18]:
data = [0,10,20,30];
js_text = js_text_template.substitute({'data': json.dumps(data)})
HTML(html_template.substitute({'js_text': js_text}))


Out[18]:

Only run the following cell after running everything else. It was only places here so the slider could be next to the visualization.


In [36]:
interact(update_from_slider, x=widgets.IntSlider(min=0,max=len(data)-10,step=1,value=0));


OK, the D3 area is set up

Now we'll focus on live updating. A manual test first.


In [25]:
js_text_template_2 = Template('''
data = $data;
update();
console.log("updating");
''')

In [26]:
data = [0,0,0,0,0,0]

def update_graph(data):
    js_text = js_text_template_2.substitute({'data': json.dumps(data)})
    display(HTML('<script>' + js_text + '</script>'))

update_graph(data)


Step 2: Now use MQTT to update the graph

Now for the fun stuff. Using the update_graph(data) function set up above. Each group of data, separated by "start" messages, is saved in the tests array. As new data comes in, the live data graph is updated to show data from the last test. All data is neatly saved inside the tests array for replotting and further analysis later.

Using the CloudMQTT free Cat plan:


In [29]:
import paho.mqtt.client as mqtt

n_to_save = 10
data = [0 for i in range(n_to_save)]

# update_graph(data)

def on_connect(client, userdata, flags, rc):
    print("Connected with result code "+str(rc))
    client.subscribe("/outTopic")

def on_message(client, userdata, msg):
    global data
    try:
        msg_json = json.loads(msg.payload)
    except:
        print("Error")
        print(msg.topic+" "+str(msg.payload))
        return
    print(msg_json)
    if msg_json['type'] == "BINARY" and msg_json['label'] == "Or":
        # data.append((msg_json['X'], msg_json['Y'], msg_json['Z']))
        data.append(msg_json['Z'])
        update_graph(data[-n_to_save:])
        print(data[-6:])

client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message
client.username_pw_set("zettlmtm", "VOUbRcmhjffA")
client.connect("m11.cloudmqtt.com", 19280, 60)
client.loop_start()
# Make sure to call client.loop_stop() later


Connected with result code 0
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2045918', u'type': u'BINARY'}
[0, 0, 0, 0, 0, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2046122', u'type': u'BINARY'}
[0, 0, 0, 0, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2046327', u'type': u'BINARY'}
[0, 0, 0, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2046530', u'type': u'BINARY'}
[0, 0, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2046735', u'type': u'BINARY'}
[0, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2046940', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2047143', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2047348', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2047552', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2047756', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2047962', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2048165', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 1.19, u'label': u'Or', u'Y': 0.0, u'X': 348.12, u'tick': u'2048371', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 1.19]
{u'Z': 0.69, u'label': u'Or', u'Y': 0.0, u'X': 348.19, u'tick': u'2048574', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 1.19, 0.69]
{u'Z': -3.31, u'label': u'Or', u'Y': 0.25, u'X': 348.69, u'tick': u'2048779', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 1.19, 0.69, -3.31]
{u'Z': -13.19, u'label': u'Or', u'Y': 0.81, u'X': 348.94, u'tick': u'2048984', u'type': u'BINARY'}
[1.19, 1.19, 1.19, 0.69, -3.31, -13.19]
{u'Z': -20.94, u'label': u'Or', u'Y': 1.0, u'X': 348.94, u'tick': u'2049187', u'type': u'BINARY'}
[1.19, 1.19, 0.69, -3.31, -13.19, -20.94]
{u'Z': -27.94, u'label': u'Or', u'Y': 0.69, u'X': 349.06, u'tick': u'2049393', u'type': u'BINARY'}
[1.19, 0.69, -3.31, -13.19, -20.94, -27.94]
{u'Z': -35.81, u'label': u'Or', u'Y': 0.31, u'X': 349.31, u'tick': u'2049597', u'type': u'BINARY'}
[0.69, -3.31, -13.19, -20.94, -27.94, -35.81]
{u'Z': -38.56, u'label': u'Or', u'Y': 0.19, u'X': 349.38, u'tick': u'2049800', u'type': u'BINARY'}
[-3.31, -13.19, -20.94, -27.94, -35.81, -38.56]
{u'Z': -39.88, u'label': u'Or', u'Y': 0.19, u'X': 349.38, u'tick': u'2050006', u'type': u'BINARY'}
[-13.19, -20.94, -27.94, -35.81, -38.56, -39.88]
{u'Z': -42.94, u'label': u'Or', u'Y': 0.0, u'X': 349.38, u'tick': u'2050209', u'type': u'BINARY'}
[-20.94, -27.94, -35.81, -38.56, -39.88, -42.94]
{u'Z': -44.19, u'label': u'Or', u'Y': 0.0, u'X': 349.44, u'tick': u'2050413', u'type': u'BINARY'}
[-27.94, -35.81, -38.56, -39.88, -42.94, -44.19]
{u'Z': -45.75, u'label': u'Or', u'Y': 0.06, u'X': 349.38, u'tick': u'2050617', u'type': u'BINARY'}
[-35.81, -38.56, -39.88, -42.94, -44.19, -45.75]
{u'Z': -42.88, u'label': u'Or', u'Y': 0.06, u'X': 349.44, u'tick': u'2050821', u'type': u'BINARY'}
[-38.56, -39.88, -42.94, -44.19, -45.75, -42.88]
{u'Z': -30.56, u'label': u'Or', u'Y': 0.81, u'X': 349.31, u'tick': u'2051025', u'type': u'BINARY'}
[-39.88, -42.94, -44.19, -45.75, -42.88, -30.56]
{u'Z': -17.06, u'label': u'Or', u'Y': 1.25, u'X': 348.31, u'tick': u'2051229', u'type': u'BINARY'}
[-42.94, -44.19, -45.75, -42.88, -30.56, -17.06]
{u'Z': -9.56, u'label': u'Or', u'Y': 0.81, u'X': 348.25, u'tick': u'2051433', u'type': u'BINARY'}
[-44.19, -45.75, -42.88, -30.56, -17.06, -9.56]
{u'Z': -3.75, u'label': u'Or', u'Y': 0.5, u'X': 348.69, u'tick': u'2051638', u'type': u'BINARY'}
[-45.75, -42.88, -30.56, -17.06, -9.56, -3.75]
{u'Z': 2.0, u'label': u'Or', u'Y': 0.19, u'X': 349.38, u'tick': u'2051843', u'type': u'BINARY'}
[-42.88, -30.56, -17.06, -9.56, -3.75, 2.0]
{u'Z': 7.56, u'label': u'Or', u'Y': 0.19, u'X': 349.44, u'tick': u'2052046', u'type': u'BINARY'}
[-30.56, -17.06, -9.56, -3.75, 2.0, 7.56]
{u'Z': 19.44, u'label': u'Or', u'Y': 0.12, u'X': 349.56, u'tick': u'2052252', u'type': u'BINARY'}
[-17.06, -9.56, -3.75, 2.0, 7.56, 19.44]
{u'Z': 32.81, u'label': u'Or', u'Y': 0.0, u'X': 350.38, u'tick': u'2052455', u'type': u'BINARY'}
[-9.56, -3.75, 2.0, 7.56, 19.44, 32.81]
{u'Z': 43.81, u'label': u'Or', u'Y': 0.31, u'X': 352.06, u'tick': u'2052660', u'type': u'BINARY'}
[-3.75, 2.0, 7.56, 19.44, 32.81, 43.81]
{u'Z': 57.19, u'label': u'Or', u'Y': 0.88, u'X': 353.38, u'tick': u'2052865', u'type': u'BINARY'}
[2.0, 7.56, 19.44, 32.81, 43.81, 57.19]
{u'Z': 69.88, u'label': u'Or', u'Y': 1.06, u'X': 354.31, u'tick': u'2053069', u'type': u'BINARY'}
[7.56, 19.44, 32.81, 43.81, 57.19, 69.88]
{u'Z': 79.25, u'label': u'Or', u'Y': 1.25, u'X': 354.69, u'tick': u'2053273', u'type': u'BINARY'}
[19.44, 32.81, 43.81, 57.19, 69.88, 79.25]
{u'Z': 88.88, u'label': u'Or', u'Y': 1.5, u'X': 351.62, u'tick': u'2053477', u'type': u'BINARY'}
[32.81, 43.81, 57.19, 69.88, 79.25, 88.88]
{u'Z': 91.06, u'label': u'Or', u'Y': 1.5, u'X': 351.0, u'tick': u'2053681', u'type': u'BINARY'}
[43.81, 57.19, 69.88, 79.25, 88.88, 91.06]
{u'Z': 91.12, u'label': u'Or', u'Y': 1.44, u'X': 350.69, u'tick': u'2053885', u'type': u'BINARY'}
[57.19, 69.88, 79.25, 88.88, 91.06, 91.12]
{u'Z': 89.0, u'label': u'Or', u'Y': 1.38, u'X': 350.56, u'tick': u'2054089', u'type': u'BINARY'}
[69.88, 79.25, 88.88, 91.06, 91.12, 89.0]
{u'Z': 84.75, u'label': u'Or', u'Y': 1.19, u'X': 350.31, u'tick': u'2054293', u'type': u'BINARY'}
[79.25, 88.88, 91.06, 91.12, 89.0, 84.75]
{u'Z': 76.69, u'label': u'Or', u'Y': 1.0, u'X': 349.19, u'tick': u'2054499', u'type': u'BINARY'}
[88.88, 91.06, 91.12, 89.0, 84.75, 76.69]
{u'Z': 66.81, u'label': u'Or', u'Y': 0.81, u'X': 348.5, u'tick': u'2054702', u'type': u'BINARY'}
[91.06, 91.12, 89.0, 84.75, 76.69, 66.81]
{u'Z': 56.94, u'label': u'Or', u'Y': 0.69, u'X': 348.25, u'tick': u'2054907', u'type': u'BINARY'}
[91.12, 89.0, 84.75, 76.69, 66.81, 56.94]
{u'Z': 47.31, u'label': u'Or', u'Y': 0.38, u'X': 348.19, u'tick': u'2055111', u'type': u'BINARY'}
[89.0, 84.75, 76.69, 66.81, 56.94, 47.31]
{u'Z': 38.5, u'label': u'Or', u'Y': 0.06, u'X': 347.88, u'tick': u'2055315', u'type': u'BINARY'}
[84.75, 76.69, 66.81, 56.94, 47.31, 38.5]
{u'Z': 30.38, u'label': u'Or', u'Y': -0.06, u'X': 347.56, u'tick': u'2055521', u'type': u'BINARY'}
[76.69, 66.81, 56.94, 47.31, 38.5, 30.38]
{u'Z': 22.88, u'label': u'Or', u'Y': -0.06, u'X': 347.62, u'tick': u'2055724', u'type': u'BINARY'}
[66.81, 56.94, 47.31, 38.5, 30.38, 22.88]
{u'Z': 10.81, u'label': u'Or', u'Y': 0.0, u'X': 349.5, u'tick': u'2055928', u'type': u'BINARY'}
[56.94, 47.31, 38.5, 30.38, 22.88, 10.81]
{u'Z': 4.06, u'label': u'Or', u'Y': 0.12, u'X': 349.88, u'tick': u'2056132', u'type': u'BINARY'}
[47.31, 38.5, 30.38, 22.88, 10.81, 4.06]
{u'Z': 3.56, u'label': u'Or', u'Y': 0.12, u'X': 350.56, u'tick': u'2056336', u'type': u'BINARY'}
[38.5, 30.38, 22.88, 10.81, 4.06, 3.56]
{u'Z': 2.12, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2056541', u'type': u'BINARY'}
[30.38, 22.88, 10.81, 4.06, 3.56, 2.12]
{u'Z': 2.06, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2056746', u'type': u'BINARY'}
[22.88, 10.81, 4.06, 3.56, 2.12, 2.06]
{u'Z': 1.94, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2056949', u'type': u'BINARY'}
[10.81, 4.06, 3.56, 2.12, 2.06, 1.94]
{u'Z': 1.88, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2057155', u'type': u'BINARY'}
[4.06, 3.56, 2.12, 2.06, 1.94, 1.88]
{u'Z': 1.81, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2057358', u'type': u'BINARY'}
[3.56, 2.12, 2.06, 1.94, 1.88, 1.81]
{u'Z': 1.75, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2057563', u'type': u'BINARY'}
[2.12, 2.06, 1.94, 1.88, 1.81, 1.75]
{u'Z': 1.75, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2057767', u'type': u'BINARY'}
[2.06, 1.94, 1.88, 1.81, 1.75, 1.75]
{u'Z': 1.69, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2057971', u'type': u'BINARY'}
[1.94, 1.88, 1.81, 1.75, 1.75, 1.69]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2058176', u'type': u'BINARY'}
[1.88, 1.81, 1.75, 1.75, 1.69, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2058380', u'type': u'BINARY'}
[1.81, 1.75, 1.75, 1.69, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2058584', u'type': u'BINARY'}
[1.75, 1.75, 1.69, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2058790', u'type': u'BINARY'}
[1.75, 1.69, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2058993', u'type': u'BINARY'}
[1.69, 1.62, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2059199', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2059403', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2059606', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2059812', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2060015', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2060221', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.62, u'label': u'Or', u'Y': 0.06, u'X': 349.31, u'tick': u'2060425', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.62, 1.62]
{u'Z': 1.56, u'label': u'Or', u'Y': 0.0, u'X': 349.31, u'tick': u'2060629', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.62, 1.56]
{u'Z': 1.56, u'label': u'Or', u'Y': 0.0, u'X': 349.31, u'tick': u'2060835', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.62, 1.56, 1.56]
{u'Z': 1.5, u'label': u'Or', u'Y': 0.0, u'X': 349.31, u'tick': u'2061038', u'type': u'BINARY'}
[1.62, 1.62, 1.62, 1.56, 1.56, 1.5]

In [31]:
client.loop_stop()


Out[31]:
3

In [33]:
print(data)


[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1.19, 1.19, 1.19, 1.19, 1.19, 1.19, 1.19, 1.19, 1.19, 1.19, 1.19, 1.19, 1.19, 0.69, -3.31, -13.19, -20.94, -27.94, -35.81, -38.56, -39.88, -42.94, -44.19, -45.75, -42.88, -30.56, -17.06, -9.56, -3.75, 2.0, 7.56, 19.44, 32.81, 43.81, 57.19, 69.88, 79.25, 88.88, 91.06, 91.12, 89.0, 84.75, 76.69, 66.81, 56.94, 47.31, 38.5, 30.38, 22.88, 10.81, 4.06, 3.56, 2.12, 2.06, 1.94, 1.88, 1.81, 1.75, 1.75, 1.69, 1.62, 1.62, 1.62, 1.62, 1.62, 1.62, 1.62, 1.62, 1.62, 1.62, 1.62, 1.62, 1.56, 1.56, 1.5, 1.5]

In [19]:
from time import sleep
for i in range(len(data)-10):
    update_graph(data[i:i+10])
    sleep(0.2)

Replaying old data!

The code in the cell below replays the old data on the graph using the collected data. This could also be a Jupyter widget to allow scrubbing if I knew how that worked.


In [34]:
from __future__ import print_function
from ipywidgets import interact, interactive, fixed
import ipywidgets as widgets

In [35]:
def update_from_slider(x):
    update_graph(data[x:x+10])

In [18]:
interact(update_from_slider, x=widgets.IntSlider(min=0,max=len(data)-10,step=1,value=0));



In [ ]: